/**
 * @author peterstone
 * March 4, 2012
 */
package cn.ac.iscas.webpage.extraction;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.select.Elements;

import cn.ac.iscas.dba.ItemDBA;
import cn.ac.iscas.dbo.Item;
import cn.ac.iscas.focuscrawler.Link;

public class WebPageExtractor extends Thread{
	private static List<ExtractRule> rules = new ArrayList<ExtractRule>();
	private static final String rulePath = "./config/Extraction_Rules.txt";
	private static File[] files;
	public static int threadNumber = 4;
	private int extractIndex = 0;
	private int extractEndIndex = 0;
	private static long startTime;
	private int id;
	ExtractRule rule;
	String name;
	Item item;
	double price;
	
	public static void init(int size, File[] dir){
		startTime = System.currentTimeMillis();
		files = dir;
		readExtractionRules(rulePath);
	}
	
	public WebPageExtractor(int id, int scale){
		int range = (int)((double)scale / threadNumber);
		extractIndex = id * range;
		
		if (id == (threadNumber - 1)){
			extractEndIndex = scale;
		} else {
			extractEndIndex = (id + 1) * range;
		}
	}
	
	public boolean isEnd(){
		if (extractIndex >= extractEndIndex) return true;
		else return false;
	}
	
	private int getIndex(){
		int index = extractIndex;
		extractIndex++;
		return index;
	}
	
	public void run(){
		while (!isEnd()){
			int index = getIndex();
			if (index % 1000 == 0){
				System.out.println(index + " : " +files[index].getAbsolutePath());
				System.out.println(System.currentTimeMillis() - startTime);
			}
    		//extractContentFromFile(files[index].getAbsolutePath());
			extractURLFromFile(files[index].getAbsolutePath());
    		//System.out.println(toString());
    		//insertIntoDB();
		}
	}

	public String toString() {
		return item.toString();
	}

	public static void readExtractionRules(String filePath) {
		try {
			BufferedReader br = new BufferedReader(new FileReader(filePath));
			String line = "";
			while ((line = br.readLine()) != null) {
				String[] tmp = line.split("=");
				rules.add(new ExtractRule(tmp[0], tmp[1]));
				line = "";
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	private void extract(Document doc){
		item = new Item();
		Iterator it = rules.iterator();
		while (it.hasNext()) {
			rule = (ExtractRule) it.next();
			
			Elements elements = doc.select(rule.content);
			Iterator<Element> it_elements = elements.iterator();
			while (it_elements.hasNext()) {				
				Element e = it_elements.next();
				System.out.println(rule.type+":"+e.text());
				List<Node> nodes = e.childNodes();
				
				if (nodes == null) {
					item.set(rule.type, e.text());
					
				} else {
					for (int i = 0; i < nodes.size(); i++) {
						extractNode(nodes.get(i));
					}
				}
			}
		}
	}

	public void extractContent(String url) {
		try {
			Document doc = Jsoup.connect(url).timeout(10 * 1000).get();
			extract(doc);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	
	/**
	 * @author Simon@itechs
	 */
	public void extractContentFromFile(String filepath) {
		if (!filepath.contains("html")) return;
		try {
			File input = new File(filepath); 
	        Document doc = Jsoup.parse(input,"UTF-8");
	        extract(doc);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	public void extractNode(Node node) {
		List<Node> children = node.childNodes();

		if (children == null || children.size() == 0) {
			String temp = node.toString().trim();
			if (temp.compareTo("") != 0) {
				item.set(rule.type, node.toString());
			}
			return;
		}

		for (int i = 0; i < children.size(); i++) {
			extractNode(children.get(i));
		}
	}
	
	public void extractURLFromFile(String filepath){
		if (!filepath.contains("html")) return;
		try {
			File input = new File(filepath); 
	        Document doc = Jsoup.parse(input,"UTF-8");
			Elements elements = doc.select("a");
			Iterator<Element> it_elements = elements.iterator();
			while (it_elements.hasNext()) {
				Element e = it_elements.next();
				String url = e.attr("abs:href");
				if (url != "")
					System.out.println(url);
			}
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

	}
	
	public void insertIntoDB(){
		new ItemDBA().storeItemDB(item);
	}
}
